# !/python/TextResuming/util/is_parser.py
# this file is used for parsing text to sentence

import yaml
import nltk.data

def main():
    print'this is main function'
    content = readYAML('20111112_theStarBiz_2.yml')
    sentences = parseText2Sentences(content)
    i = 1;
    for sentence in sentences:
        print '[%d]' % i
        i += 1
        #   print sentence
        print
    print len(sentences)
    

def readYAML(yamlPathFile):
    '''Return dictionary from a YAML file.'''
    f = open(yamlPathFile)
    dict = yaml.load(f)
    if (dict.has_key('content')):
        return dict['content']
    else:
        print 'The key content didnt found.'
        return None

def parseText2Sentences(text):
    '''Parse Text to Sentence.

        Return a list of sentences.
        The data for tokenizer must be downloaded.
        It is punkt.'''
    tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    return tokenizer.tokenize(text)

def parseText2Paragraph(text):
    '''Parse a text to some paragraphs.

        Return a list of paragraphs.
        Paragraph is defining by <p> </p> tag.'''
    text = text.replace('</p>', '<p>')
    text = text.replace('<br>', '<p>')
    text = text.replace('<br />', '<p>')
    text = text.replace('<br/>', '<p>')
    listParagraph = text.split('<p>')
    retval =  []
    for para in listParagraph:
        if (para == ''):
            pass
        else:
            retval.append(para)
    return retval

def readTitleLinkNews(text):
    '''Read a text file.

        1st line is the title.
        2nd line is the source link.
        the rest is the content.
        It will return a list represent a news'''
    try:
        fileInput = text
        news = open(fileInput)
        newsList = news.readlines()
        #print newsList[0]
        retval = []
        retval.append(newsList[0])
        retval.append(newsList[1])
        contentList = list(newsList[2:len(newsList)])
        retval.append(parseText2Sentences(''.join(contentList)))
        #printAyuList(retval[2])
        #print len(retval[2])
        news.close()
        return retval
    except Exception, e:
        print 'Something error in readTitleLinkNews',e
        return None

def printAyuList(list):
    i = 1
    for elmt in list:
        print i, elmt
        i += 1

def tesText2Paragraph():
    test = '<p>Cinemas, cafes, shops and banks were set ablaze in central Athens and black-masked protesters fought riot police outside parliament before lawmakers voted on the package that demands deep pay, pension and job cuts - the price of a 130 billion euro bailout needed to keep the country afloat.</p><p>State television reported the violence spread to the tourist islands of Corfu and Crete, the northern city of Thessaloniki and towns in central Greece. Police said 150 shops were looted in the capital and 34 buildings set ablaze.</p><p>Altogether 199 of the 300 lawmakers backed the bill, but 43 deputies from the two parties in the government of Prime Minister Lucas Papademos, the socialists and conservatives, rebelled by voting against. They were immediately expelled by their parties.</p><p>The rebellion and street violence foreshadowed the problems the government faces in implementing the cuts, which include a 22 percent reduction in the minimum wage - a package critics say condemns the Greek economy to an ever-deeper downward spiral.</p><p>Papademos, a technocrat brought into get a grip on his country\'s crisis, denounced the worst breakdown of order since 2008 when violence gripped Greece for weeks after police shot a 15-year-old schoolboy.</p><p>"Vandalism, violence and destruction have no place in a democratic country and won\'t be tolerated," he told parliament as it prepared to vote on the new 130 billion euro bailout to save Greece from a chaotic bankruptcy.</p><p>"SHORT-TERM SACRIFICES"</p><p>But he admitted that imposing the austerity on a nation that has already endured several years of cuts would be tough.</p><p>"Ahead of us, we have a complete and credible economic program to exit the fiscal and economic crisis. It is a program which safeguards, more than anything else, the country\'s place in the euro," he said.</p><p>"The full, timely and effective implementation of the program won\'t be easy. We are fully aware that the economic program means short term sacrifices for the Greek people."</p><p>Greece needs the international funds before March 20 to meet debt repayments of 14.5 billion euros, or suffer a chaotic default which could shake the entire euro zone.</p><p>Outside parliament chaos reigned. A Reuters photographer saw buildings in Athens engulfed in flames and huge plumes of smoke rose in the night sky.</p><p>"We are facing destruction. Our country, our home, has become ripe for burning, the center of Athens is in flames. We cannot allow populism to burn our country down," conservative lawmaker Costis Hatzidakis told parliament.</p><p>The air in Syntagma Square outside parliament was thick with tear gas as riot police fought running battles with youths who smashed marble balustrades and hurled stones and petrol bombs.</p><p>Terrified Greeks and tourists fled the rock-strewn streets and the clouds of stinging gas, cramming into hotel lobbies for shelter as lines of riot police struggled to contain the mayhem.</p><p>State NET television reported that trouble had also broken out in Heraklion, capital of Crete, as well as the towns of Volos and Agrinio in central Greece.</p><p>On the streets of Athens many businesses were ablaze, including the neo-classical home to the Attikon cinema dating from 1870 and a building housing the Asty, an underground cinema used by the Gestapo during World War Two as a torture chamber.</p><p>As fighting raged for hours, protesters threw bombs made from gas canisters as riot police advanced across the square on the crowds, firing tear gas and stun grenades. Loud booms from the protests could be heard inside parliament.</p><p>^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</p><p>Interactive timeline link.reuters.com/muq56s</p><p>Euro zone crisis in graphics r.reuters.com/hyb65p</p><p>^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^</p><p>NO GOOD CHOICES</p><p>Before the vote, Finance Minister Evangelos Venizelos told parliament that the alternative to the international bailout - bankruptcy and a departure from the euro zone - would be far worse for Greeks.</p><p>"The choice is not between sacrifice and no sacrifices at all, but between sacrifices and unimaginably harsher ones," he told a stormy debate.</p><p>The EU and IMF say they have had enough of broken promises and that the funds will be released only with the clear commitment of Greek political leaders that they will implement the reforms whoever wins an election potentially in April.</p><p>Euro zone paymaster Germany ratcheted up the pressure on Sunday. "The promises from Greece aren\'t enough for us any more," German Finance Minister Wolfgang Schaeuble said in an interview published Sunday in Welt am Sonntag newspaper.</p><p>"Greece needs to do its own homework to become competitive, whether that happens in conjunction with a new rescue program or by another route that we actually don\'t want to take," he said.</p><p>When asked if that other route meant Greece quitting the euro zone, Schaeuble said: "That is all in the hands of the Greeks themselves. But even in the event (Greece leaves the euro zone), which almost no one assumes will happen, they will still remain part of Europe."</p><p>The bill sets out 3.3 billion euros ($4.35 billion) of extra budget cuts for this year alone.</p><p>It also provides for a bond swap to ease Greece\'s debt burden by cutting the real value of private-sector investors\' bond holdings by some 70 percent. Greece would have missed a February 17 deadline to offer a debt "haircut" to private bondholders if the vote had not been passed.</p><p>Many Greeks believe their living standards are collapsing already and the new measures will only deepen their misery.</p><p>"Enough is enough!" said 89-year-old Manolis Glezos, one of Greece\'s most famous leftists. "They have no idea what an uprising by the Greek people means. And the Greek people, regardless of ideology, have risen."</p><p>Glezos is a national hero for sneaking up the Acropolis at night in 1941 and tearing down a Nazi flag from under the noses of the German occupiers, raising the morale of Athens residents.</p><p>"These measures of annihilation will not pass," Glezos said on Syntagma Square, visibly overcome by teargas and holding a mask over his mouth.</p>'
    retval = parseText2Paragraph(test)
    i = 1
    for r in retval:
        print i
        i += 1
        print r

    print 'end of test'

if __name__ == '__main__':
    #main()
    a = readTitleLinkNews('C://Python27/TextResuming/test/10News/news1.txt')
    printAyuList(a)
